{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: nn\n",
    "-----------\n",
    "\n",
    "A fully-connected ReLU network with one hidden layer, trained to predict y from x\n",
    "by minimizing squared Euclidean distance.\n",
    "\n",
    "This implementation uses the nn package from PyTorch to build the network.\n",
    "PyTorch autograd makes it easy to define computational graphs and take gradients,\n",
    "<p style=\"color:red\">but raw autograd can be a bit too low-level for defining complex neural networks;\n",
    "this is where the nn package can help.</p> The nn package defines a set of Modules,\n",
    "which you can think of as <strong style=\"color:red\">a neural network layer that has produces output from\n",
    "input and may have some trainable weights.</strong>\n",
    "\n",
    "Source Link: http://pytorch.org/tutorials/beginner/examples_nn/two_layer_net_nn.html\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Orinal Tutorial code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 732.9182739257812\n",
      "1 680.7958374023438\n",
      "2 635.081787109375\n",
      "3 594.8429565429688\n",
      "4 558.852783203125\n",
      "5 526.3436889648438\n",
      "6 496.7332458496094\n",
      "7 469.5028381347656\n",
      "8 444.3468933105469\n",
      "9 421.0668029785156\n",
      "10 399.3390197753906\n",
      "11 378.8065185546875\n",
      "12 359.3966064453125\n",
      "13 340.9505920410156\n",
      "14 323.32647705078125\n",
      "15 306.4247131347656\n",
      "16 290.30389404296875\n",
      "17 274.99212646484375\n",
      "18 260.3421630859375\n",
      "19 246.3540802001953\n",
      "20 233.0052032470703\n",
      "21 220.28074645996094\n",
      "22 208.1381378173828\n",
      "23 196.60569763183594\n",
      "24 185.6157989501953\n",
      "25 175.1521759033203\n",
      "26 165.13548278808594\n",
      "27 155.63172912597656\n",
      "28 146.5999298095703\n",
      "29 137.99583435058594\n",
      "30 129.85894775390625\n",
      "31 122.16447448730469\n",
      "32 114.89279174804688\n",
      "33 108.02578735351562\n",
      "34 101.55492401123047\n",
      "35 95.45748138427734\n",
      "36 89.71915435791016\n",
      "37 84.30313873291016\n",
      "38 79.21401977539062\n",
      "39 74.43704223632812\n",
      "40 69.9603042602539\n",
      "41 65.76519775390625\n",
      "42 61.8353271484375\n",
      "43 58.152671813964844\n",
      "44 54.695621490478516\n",
      "45 51.45276641845703\n",
      "46 48.41431427001953\n",
      "47 45.57013702392578\n",
      "48 42.90782165527344\n",
      "49 40.41434097290039\n",
      "50 38.07864761352539\n",
      "51 35.88711166381836\n",
      "52 33.829010009765625\n",
      "53 31.90105628967285\n",
      "54 30.09281349182129\n",
      "55 28.395545959472656\n",
      "56 26.799053192138672\n",
      "57 25.30072593688965\n",
      "58 23.8935604095459\n",
      "59 22.571828842163086\n",
      "60 21.32986068725586\n",
      "61 20.161123275756836\n",
      "62 19.063323974609375\n",
      "63 18.030847549438477\n",
      "64 17.06005096435547\n",
      "65 16.14588737487793\n",
      "66 15.28408432006836\n",
      "67 14.471415519714355\n",
      "68 13.706422805786133\n",
      "69 12.985347747802734\n",
      "70 12.305042266845703\n",
      "71 11.663025856018066\n",
      "72 11.057016372680664\n",
      "73 10.484746932983398\n",
      "74 9.944375991821289\n",
      "75 9.433773040771484\n",
      "76 8.950862884521484\n",
      "77 8.49421215057373\n",
      "78 8.06224536895752\n",
      "79 7.653835296630859\n",
      "80 7.267484664916992\n",
      "81 6.901413917541504\n",
      "82 6.553553581237793\n",
      "83 6.224673748016357\n",
      "84 5.9125075340271\n",
      "85 5.617003440856934\n",
      "86 5.337141036987305\n",
      "87 5.072457313537598\n",
      "88 4.8220319747924805\n",
      "89 4.584941387176514\n",
      "90 4.360015869140625\n",
      "91 4.146988391876221\n",
      "92 3.945094108581543\n",
      "93 3.7533974647521973\n",
      "94 3.5715715885162354\n",
      "95 3.399153470993042\n",
      "96 3.2357470989227295\n",
      "97 3.0808956623077393\n",
      "98 2.9337658882141113\n",
      "99 2.794316530227661\n",
      "100 2.6618711948394775\n",
      "101 2.536137580871582\n",
      "102 2.4168004989624023\n",
      "103 2.3031609058380127\n",
      "104 2.1953063011169434\n",
      "105 2.092968225479126\n",
      "106 1.9956295490264893\n",
      "107 1.9032208919525146\n",
      "108 1.8153892755508423\n",
      "109 1.731921672821045\n",
      "110 1.652485728263855\n",
      "111 1.576965570449829\n",
      "112 1.5052807331085205\n",
      "113 1.4371665716171265\n",
      "114 1.37230384349823\n",
      "115 1.3105813264846802\n",
      "116 1.251875638961792\n",
      "117 1.1959972381591797\n",
      "118 1.142863154411316\n",
      "119 1.0922067165374756\n",
      "120 1.0439589023590088\n",
      "121 0.9979961514472961\n",
      "122 0.9541940093040466\n",
      "123 0.9124483466148376\n",
      "124 0.8727037310600281\n",
      "125 0.834831714630127\n",
      "126 0.7987141609191895\n",
      "127 0.7642777562141418\n",
      "128 0.7314593195915222\n",
      "129 0.7001343369483948\n",
      "130 0.6702470779418945\n",
      "131 0.6417502760887146\n",
      "132 0.6145665049552917\n",
      "133 0.5886298418045044\n",
      "134 0.563971757888794\n",
      "135 0.5404325723648071\n",
      "136 0.5179718732833862\n",
      "137 0.49650493264198303\n",
      "138 0.47602036595344543\n",
      "139 0.45644569396972656\n",
      "140 0.43774324655532837\n",
      "141 0.41985610127449036\n",
      "142 0.4027687609195709\n",
      "143 0.3864264190196991\n",
      "144 0.37078869342803955\n",
      "145 0.3558444678783417\n",
      "146 0.3414990305900574\n",
      "147 0.32777202129364014\n",
      "148 0.31463533639907837\n",
      "149 0.3020881414413452\n",
      "150 0.29009565711021423\n",
      "151 0.2786107361316681\n",
      "152 0.26762017607688904\n",
      "153 0.2570899426937103\n",
      "154 0.24701222777366638\n",
      "155 0.23737013339996338\n",
      "156 0.22812138497829437\n",
      "157 0.2192552238702774\n",
      "158 0.21076422929763794\n",
      "159 0.20263032615184784\n",
      "160 0.19482918083667755\n",
      "161 0.18735109269618988\n",
      "162 0.180190309882164\n",
      "163 0.17332428693771362\n",
      "164 0.16673798859119415\n",
      "165 0.16042445600032806\n",
      "166 0.1543661504983902\n",
      "167 0.14855794608592987\n",
      "168 0.14299187064170837\n",
      "169 0.1376449018716812\n",
      "170 0.13250894844532013\n",
      "171 0.1275733858346939\n",
      "172 0.12283024936914444\n",
      "173 0.11827965825796127\n",
      "174 0.11390659213066101\n",
      "175 0.10971007496118546\n",
      "176 0.10567913204431534\n",
      "177 0.10180911421775818\n",
      "178 0.09809145331382751\n",
      "179 0.09451711177825928\n",
      "180 0.09108477085828781\n",
      "181 0.08778468519449234\n",
      "182 0.0846121609210968\n",
      "183 0.08156493306159973\n",
      "184 0.07863251864910126\n",
      "185 0.07581031322479248\n",
      "186 0.07309716939926147\n",
      "187 0.07049061357975006\n",
      "188 0.06798256933689117\n",
      "189 0.06556525081396103\n",
      "190 0.06323979794979095\n",
      "191 0.06100303307175636\n",
      "192 0.05885102227330208\n",
      "193 0.056779779493808746\n",
      "194 0.054786454886198044\n",
      "195 0.052867159247398376\n",
      "196 0.05101766437292099\n",
      "197 0.04923722520470619\n",
      "198 0.04752300679683685\n",
      "199 0.04587104544043541\n",
      "200 0.04428377375006676\n",
      "201 0.04275226965546608\n",
      "202 0.04127780348062515\n",
      "203 0.039857879281044006\n",
      "204 0.03848773613572121\n",
      "205 0.037167664617300034\n",
      "206 0.03589527681469917\n",
      "207 0.03466936573386192\n",
      "208 0.03348740562796593\n",
      "209 0.03234894946217537\n",
      "210 0.03124951384961605\n",
      "211 0.030189212411642075\n",
      "212 0.029167987406253815\n",
      "213 0.028183573856949806\n",
      "214 0.02723313495516777\n",
      "215 0.026315761730074883\n",
      "216 0.025431448593735695\n",
      "217 0.02457796409726143\n",
      "218 0.023755047470331192\n",
      "219 0.022961076349020004\n",
      "220 0.022194307297468185\n",
      "221 0.02145431563258171\n",
      "222 0.020740097388625145\n",
      "223 0.020052431151270866\n",
      "224 0.019388213753700256\n",
      "225 0.01874646171927452\n",
      "226 0.01812686398625374\n",
      "227 0.01752837747335434\n",
      "228 0.01695067621767521\n",
      "229 0.01639258861541748\n",
      "230 0.0158541277050972\n",
      "231 0.015334423631429672\n",
      "232 0.014831949956715107\n",
      "233 0.014346582815051079\n",
      "234 0.013878353871405125\n",
      "235 0.013425646349787712\n",
      "236 0.01298852264881134\n",
      "237 0.012565975077450275\n",
      "238 0.012157527729868889\n",
      "239 0.01176275871694088\n",
      "240 0.011381275951862335\n",
      "241 0.011012946255505085\n",
      "242 0.010657090693712234\n",
      "243 0.010312935337424278\n",
      "244 0.009980625472962856\n",
      "245 0.009659322910010815\n",
      "246 0.009348621591925621\n",
      "247 0.00904841534793377\n",
      "248 0.00875804852694273\n",
      "249 0.008477451279759407\n",
      "250 0.00820603221654892\n",
      "251 0.007943709380924702\n",
      "252 0.007690083235502243\n",
      "253 0.007444922346621752\n",
      "254 0.007207589689642191\n",
      "255 0.0069783348590135574\n",
      "256 0.006756673566997051\n",
      "257 0.0065421247854828835\n",
      "258 0.006335028912872076\n",
      "259 0.006134669296443462\n",
      "260 0.005940721835941076\n",
      "261 0.005753021687269211\n",
      "262 0.005571658723056316\n",
      "263 0.005396339576691389\n",
      "264 0.005226705223321915\n",
      "265 0.005062376148998737\n",
      "266 0.0049034785479307175\n",
      "267 0.00474970368668437\n",
      "268 0.00460087601095438\n",
      "269 0.004456854425370693\n",
      "270 0.0043173241429030895\n",
      "271 0.004182298667728901\n",
      "272 0.004051665309816599\n",
      "273 0.003925271797925234\n",
      "274 0.003803088329732418\n",
      "275 0.0036845800932496786\n",
      "276 0.0035698821302503347\n",
      "277 0.0034588822163641453\n",
      "278 0.003351400373503566\n",
      "279 0.003247424028813839\n",
      "280 0.0031467226799577475\n",
      "281 0.0030492122750729322\n",
      "282 0.0029548010788857937\n",
      "283 0.0028633628971874714\n",
      "284 0.002774932887405157\n",
      "285 0.0026892549358308315\n",
      "286 0.0026062754914164543\n",
      "287 0.0025259158574044704\n",
      "288 0.002448051003739238\n",
      "289 0.002372682560235262\n",
      "290 0.0022998268250375986\n",
      "291 0.0022291464265435934\n",
      "292 0.002160649048164487\n",
      "293 0.002094342140480876\n",
      "294 0.0020302007906138897\n",
      "295 0.0019680156838148832\n",
      "296 0.0019077161559835076\n",
      "297 0.0018493619281798601\n",
      "298 0.0017928143497556448\n",
      "299 0.0017380104400217533\n",
      "300 0.001684935879893601\n",
      "301 0.001633505686186254\n",
      "302 0.0015836815582588315\n",
      "303 0.0015353894559666514\n",
      "304 0.0014886496355757117\n",
      "305 0.0014433719916269183\n",
      "306 0.0013994459295645356\n",
      "307 0.0013568646972998977\n",
      "308 0.0013156419154256582\n",
      "309 0.0012757045915350318\n",
      "310 0.0012369976611807942\n",
      "311 0.0011994760716333985\n",
      "312 0.0011631109518930316\n",
      "313 0.0011278643505647779\n",
      "314 0.0010937157785519958\n",
      "315 0.0010606818832457066\n",
      "316 0.001028583967126906\n",
      "317 0.0009974922286346555\n",
      "318 0.0009673430467955768\n",
      "319 0.0009381462004967034\n",
      "320 0.0009098385344259441\n",
      "321 0.000882404507137835\n",
      "322 0.0008558072149753571\n",
      "323 0.0008300188928842545\n",
      "324 0.0008050501928664744\n",
      "325 0.0007808395894244313\n",
      "326 0.0007573362672701478\n",
      "327 0.0007345689227804542\n",
      "328 0.000712492736056447\n",
      "329 0.0006911054369993508\n",
      "330 0.0006703946855850518\n",
      "331 0.000650281086564064\n",
      "332 0.0006307871080935001\n",
      "333 0.0006118844030424953\n",
      "334 0.0005935726221650839\n",
      "335 0.0005758015322498977\n",
      "336 0.0005585704348050058\n",
      "337 0.000541851797606796\n",
      "338 0.0005256658187136054\n",
      "339 0.0005099552217870951\n",
      "340 0.0004947145935148001\n",
      "341 0.000479961687233299\n",
      "342 0.00046563398791477084\n",
      "343 0.0004517543420661241\n",
      "344 0.00043828264460898936\n",
      "345 0.00042520856368355453\n",
      "346 0.00041254478855989873\n",
      "347 0.00040025272755883634\n",
      "348 0.0003883421595674008\n",
      "349 0.0003767844755202532\n",
      "350 0.0003655784239526838\n",
      "351 0.00035470607690513134\n",
      "352 0.0003441687149461359\n",
      "353 0.0003339519607834518\n",
      "354 0.00032402726355940104\n",
      "355 0.0003144058573525399\n",
      "356 0.00030507458723150194\n",
      "357 0.00029603016446344554\n",
      "358 0.0002872442710213363\n",
      "359 0.0002787453995551914\n",
      "360 0.000270484306383878\n",
      "361 0.0002624756016302854\n",
      "362 0.00025469763204455376\n",
      "363 0.00024716206826269627\n",
      "364 0.00023985258303582668\n",
      "365 0.00023275957209989429\n",
      "366 0.00022588447609450668\n",
      "367 0.0002192153624491766\n",
      "368 0.00021273807215038687\n",
      "369 0.00020646174380090088\n",
      "370 0.00020037258218508214\n",
      "371 0.00019445456564426422\n",
      "372 0.0001887197868200019\n",
      "373 0.00018315250054001808\n",
      "374 0.0001777534926077351\n",
      "375 0.00017251947429031134\n",
      "376 0.00016743828018661588\n",
      "377 0.00016250138287432492\n",
      "378 0.00015771201287861913\n",
      "379 0.0001530744630144909\n",
      "380 0.00014856949565000832\n",
      "381 0.00014419632498174906\n",
      "382 0.00013995589688420296\n",
      "383 0.0001358426088700071\n",
      "384 0.00013185525313019753\n",
      "385 0.00012797377712558955\n",
      "386 0.00012421573046594858\n",
      "387 0.0001205693042720668\n",
      "388 0.0001170300311059691\n",
      "389 0.00011359896598150954\n",
      "390 0.00011026539868908003\n",
      "391 0.0001070264115696773\n",
      "392 0.00010389478848082945\n",
      "393 0.00010084596578963101\n",
      "394 9.789545583771542e-05\n",
      "395 9.502549801254645e-05\n",
      "396 9.22429098864086e-05\n",
      "397 8.954121585702524e-05\n",
      "398 8.692459232406691e-05\n",
      "399 8.43775924295187e-05\n",
      "400 8.191120286937803e-05\n",
      "401 7.95172163634561e-05\n",
      "402 7.719077257206663e-05\n",
      "403 7.493336306652054e-05\n",
      "404 7.274381641764194e-05\n",
      "405 7.06198115949519e-05\n",
      "406 6.85549239278771e-05\n",
      "407 6.655438482994214e-05\n",
      "408 6.461251905420795e-05\n",
      "409 6.272589962463826e-05\n",
      "410 6.089633461670019e-05\n",
      "411 5.9117908676853403e-05\n",
      "412 5.739197877119295e-05\n",
      "413 5.572173176915385e-05\n",
      "414 5.409970981418155e-05\n",
      "415 5.252132541500032e-05\n",
      "416 5.098980909679085e-05\n",
      "417 4.950506263412535e-05\n",
      "418 4.8060457629617304e-05\n",
      "419 4.666161476052366e-05\n",
      "420 4.5303833758225664e-05\n",
      "421 4.398570672492497e-05\n",
      "422 4.270526915206574e-05\n",
      "423 4.146157152717933e-05\n",
      "424 4.02549521822948e-05\n",
      "425 3.9082504372345284e-05\n",
      "426 3.794826625380665e-05\n",
      "427 3.6845314753009006e-05\n",
      "428 3.577445386326872e-05\n",
      "429 3.473577453405596e-05\n",
      "430 3.372572973603383e-05\n",
      "431 3.2746476790634915e-05\n",
      "432 3.1793530069990084e-05\n",
      "433 3.0871175113134086e-05\n",
      "434 2.99752955470467e-05\n",
      "435 2.9103108317940496e-05\n",
      "436 2.8260186809347942e-05\n",
      "437 2.743964796536602e-05\n",
      "438 2.664160456333775e-05\n",
      "439 2.586844129837118e-05\n",
      "440 2.511999264243059e-05\n",
      "441 2.439194759062957e-05\n",
      "442 2.3684582629357465e-05\n",
      "443 2.2998528947937302e-05\n",
      "444 2.2331605578074232e-05\n",
      "445 2.1685482352040708e-05\n",
      "446 2.1055479010101408e-05\n",
      "447 2.0448234863579273e-05\n",
      "448 1.98541038116673e-05\n",
      "449 1.9277662431704812e-05\n",
      "450 1.8722368622547947e-05\n",
      "451 1.8180002371082082e-05\n",
      "452 1.765284287102986e-05\n",
      "453 1.7141923308372498e-05\n",
      "454 1.6644982679281384e-05\n",
      "455 1.6164851331268437e-05\n",
      "456 1.5697643902967684e-05\n",
      "457 1.5243070265569258e-05\n",
      "458 1.480365426687058e-05\n",
      "459 1.4376139006344602e-05\n",
      "460 1.3959212992631365e-05\n",
      "461 1.3555931218434125e-05\n",
      "462 1.3164404663257301e-05\n",
      "463 1.2785708349838387e-05\n",
      "464 1.2415385754138697e-05\n",
      "465 1.2057045751134865e-05\n",
      "466 1.1708070815075189e-05\n",
      "467 1.1369576895958744e-05\n",
      "468 1.1043356607842725e-05\n",
      "469 1.072274062607903e-05\n",
      "470 1.0413679774501361e-05\n",
      "471 1.0113507414644118e-05\n",
      "472 9.820608283916954e-06\n",
      "473 9.53835933614755e-06\n",
      "474 9.262301318813115e-06\n",
      "475 8.995098141895141e-06\n",
      "476 8.735409210203215e-06\n",
      "477 8.485341822961345e-06\n",
      "478 8.240641363954637e-06\n",
      "479 8.00339512352366e-06\n",
      "480 7.771929631417152e-06\n",
      "481 7.548373559984611e-06\n",
      "482 7.330593689403031e-06\n",
      "483 7.119666406651959e-06\n",
      "484 6.914940968272276e-06\n",
      "485 6.715137715218589e-06\n",
      "486 6.521251634694636e-06\n",
      "487 6.3340808083012234e-06\n",
      "488 6.151134130050195e-06\n",
      "489 5.974646228423808e-06\n",
      "490 5.803221029054839e-06\n",
      "491 5.635500201606192e-06\n",
      "492 5.473031251312932e-06\n",
      "493 5.316123861121014e-06\n",
      "494 5.162494289834285e-06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "495 5.0136750360252336e-06\n",
      "496 4.8696601879782975e-06\n",
      "497 4.729776719614165e-06\n",
      "498 4.594629899656866e-06\n",
      "499 4.461036496650195e-06\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold inputs and outputs, and wrap them in Variables.\n",
    "x = Variable(torch.randn(N, D_in))\n",
    "y = Variable(torch.randn(N, D_out), requires_grad=False)\n",
    "\n",
    "# Use the nn package to define our model as a sequence of layers. nn.Sequential\n",
    "# is a Module which contains other Modules, and applies them in sequence to\n",
    "# produce its output. Each Linear Module computes output from input using a\n",
    "# linear function, and holds internal Variables for its weight and bias.\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    ")\n",
    "\n",
    "# The nn package also contains definitions of popular loss functions; in this\n",
    "# case we will use Mean Squared Error (MSE) as our loss function.\n",
    "loss_fn = torch.nn.MSELoss(size_average=False)\n",
    "\n",
    "learning_rate = 1e-4\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y by passing x to the model. Module objects\n",
    "    # override the __call__ operator so you can call them like functions. When\n",
    "    # doing so you pass a Variable of input data to the Module and it produces\n",
    "    # a Variable of output data.\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # Compute and print loss. We pass Variables containing the predicted and true\n",
    "    # values of y, and the loss function returns a Variable containing the\n",
    "    # loss.\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Zero the gradients before running the backward pass.\n",
    "    model.zero_grad()\n",
    "\n",
    "    # Backward pass: compute gradient of the loss with respect to all the learnable\n",
    "    # parameters of the model. Internally, the parameters of each Module are stored\n",
    "    # in Variables with requires_grad=True, so this call will compute gradients for\n",
    "    # all learnable parameters in the model.\n",
    "    loss.backward()\n",
    "\n",
    "    # Update the weights using gradient descent. Each parameter is a Variable, so\n",
    "    # we can access its data and gradients like we did before.\n",
    "    for param in model.parameters():\n",
    "        param.data -= learning_rate * param.grad.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Without #annotation Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 643.7648315429688\n",
      "1 594.0890502929688\n",
      "2 551.3575439453125\n",
      "3 513.9000244140625\n",
      "4 480.87255859375\n",
      "5 451.5255126953125\n",
      "6 424.8398132324219\n",
      "7 400.4261169433594\n",
      "8 377.9382019042969\n",
      "9 357.271728515625\n",
      "10 338.1226501464844\n",
      "11 320.20574951171875\n",
      "12 303.2518310546875\n",
      "13 287.2070007324219\n",
      "14 271.9260559082031\n",
      "15 257.4627685546875\n",
      "16 243.72425842285156\n",
      "17 230.61788940429688\n",
      "18 218.16448974609375\n",
      "19 206.342041015625\n",
      "20 195.05458068847656\n",
      "21 184.2813720703125\n",
      "22 174.0470733642578\n",
      "23 164.29884338378906\n",
      "24 155.0045928955078\n",
      "25 146.16156005859375\n",
      "26 137.74839782714844\n",
      "27 129.75119018554688\n",
      "28 122.14928436279297\n",
      "29 114.9527816772461\n",
      "30 108.11542510986328\n",
      "31 101.63982391357422\n",
      "32 95.50341033935547\n",
      "33 89.70565032958984\n",
      "34 84.22037506103516\n",
      "35 79.05567169189453\n",
      "36 74.1976547241211\n",
      "37 69.62667083740234\n",
      "38 65.328369140625\n",
      "39 61.296504974365234\n",
      "40 57.507511138916016\n",
      "41 53.94853591918945\n",
      "42 50.605770111083984\n",
      "43 47.464569091796875\n",
      "44 44.520782470703125\n",
      "45 41.767940521240234\n",
      "46 39.18711853027344\n",
      "47 36.76841735839844\n",
      "48 34.50346755981445\n",
      "49 32.38359832763672\n",
      "50 30.399389266967773\n",
      "51 28.54187774658203\n",
      "52 26.804718017578125\n",
      "53 25.178112030029297\n",
      "54 23.657142639160156\n",
      "55 22.232683181762695\n",
      "56 20.900455474853516\n",
      "57 19.652494430541992\n",
      "58 18.485002517700195\n",
      "59 17.392728805541992\n",
      "60 16.369836807250977\n",
      "61 15.412313461303711\n",
      "62 14.515030860900879\n",
      "63 13.673959732055664\n",
      "64 12.88329792022705\n",
      "65 12.141616821289062\n",
      "66 11.447157859802246\n",
      "67 10.794857025146484\n",
      "68 10.182958602905273\n",
      "69 9.609127044677734\n",
      "70 9.070199966430664\n",
      "71 8.564756393432617\n",
      "72 8.089903831481934\n",
      "73 7.643778324127197\n",
      "74 7.224410533905029\n",
      "75 6.83017635345459\n",
      "76 6.46004581451416\n",
      "77 6.1114420890808105\n",
      "78 5.783817291259766\n",
      "79 5.4752936363220215\n",
      "80 5.1852288246154785\n",
      "81 4.9117112159729\n",
      "82 4.653841972351074\n",
      "83 4.41071891784668\n",
      "84 4.181772232055664\n",
      "85 3.9661996364593506\n",
      "86 3.762617588043213\n",
      "87 3.5707552433013916\n",
      "88 3.389591693878174\n",
      "89 3.2186405658721924\n",
      "90 3.0571324825286865\n",
      "91 2.9044268131256104\n",
      "92 2.7602956295013428\n",
      "93 2.6238505840301514\n",
      "94 2.4949612617492676\n",
      "95 2.3729071617126465\n",
      "96 2.2576522827148438\n",
      "97 2.148618698120117\n",
      "98 2.045339822769165\n",
      "99 1.9474506378173828\n",
      "100 1.8546797037124634\n",
      "101 1.7666798830032349\n",
      "102 1.68320894241333\n",
      "103 1.603985071182251\n",
      "104 1.5288842916488647\n",
      "105 1.4576563835144043\n",
      "106 1.390121579170227\n",
      "107 1.3260611295700073\n",
      "108 1.2651783227920532\n",
      "109 1.2072969675064087\n",
      "110 1.1523735523223877\n",
      "111 1.1001402139663696\n",
      "112 1.0504417419433594\n",
      "113 1.0032479763031006\n",
      "114 0.958567202091217\n",
      "115 0.9159932732582092\n",
      "116 0.8755185604095459\n",
      "117 0.836984395980835\n",
      "118 0.8002479672431946\n",
      "119 0.7652481198310852\n",
      "120 0.7319182753562927\n",
      "121 0.7001960277557373\n",
      "122 0.6699308156967163\n",
      "123 0.6411037445068359\n",
      "124 0.6135928630828857\n",
      "125 0.5873593091964722\n",
      "126 0.5623412728309631\n",
      "127 0.538460373878479\n",
      "128 0.5156698226928711\n",
      "129 0.49392634630203247\n",
      "130 0.4731583297252655\n",
      "131 0.45333999395370483\n",
      "132 0.434410959482193\n",
      "133 0.4163289964199066\n",
      "134 0.39905208349227905\n",
      "135 0.38254061341285706\n",
      "136 0.3667714297771454\n",
      "137 0.3516993224620819\n",
      "138 0.33727124333381653\n",
      "139 0.3234710097312927\n",
      "140 0.3102799654006958\n",
      "141 0.297670841217041\n",
      "142 0.2855944037437439\n",
      "143 0.27403318881988525\n",
      "144 0.26297834515571594\n",
      "145 0.25240203738212585\n",
      "146 0.2422782927751541\n",
      "147 0.23257768154144287\n",
      "148 0.2232983559370041\n",
      "149 0.2144090235233307\n",
      "150 0.20589900016784668\n",
      "151 0.19775401055812836\n",
      "152 0.18994100391864777\n",
      "153 0.18246154487133026\n",
      "154 0.17529742419719696\n",
      "155 0.16842706501483917\n",
      "156 0.16184312105178833\n",
      "157 0.15553471446037292\n",
      "158 0.14948312938213348\n",
      "159 0.14368107914924622\n",
      "160 0.1381205916404724\n",
      "161 0.13278530538082123\n",
      "162 0.12767194211483002\n",
      "163 0.12276335805654526\n",
      "164 0.11805469542741776\n",
      "165 0.11353695392608643\n",
      "166 0.1091991737484932\n",
      "167 0.10503920912742615\n",
      "168 0.10102542489767075\n",
      "169 0.09717348963022232\n",
      "170 0.09347862005233765\n",
      "171 0.0899391695857048\n",
      "172 0.08653543144464493\n",
      "173 0.08326800912618637\n",
      "174 0.08012845367193222\n",
      "175 0.07711387425661087\n",
      "176 0.07421974092721939\n",
      "177 0.07144192606210709\n",
      "178 0.06877363473176956\n",
      "179 0.06621000915765762\n",
      "180 0.06374823302030563\n",
      "181 0.061381299048662186\n",
      "182 0.05910569429397583\n",
      "183 0.05691765993833542\n",
      "184 0.05481674522161484\n",
      "185 0.0527978390455246\n",
      "186 0.05085844174027443\n",
      "187 0.048992373049259186\n",
      "188 0.047196775674819946\n",
      "189 0.04547155648469925\n",
      "190 0.04381680116057396\n",
      "191 0.04222201183438301\n",
      "192 0.04068892076611519\n",
      "193 0.03921402618288994\n",
      "194 0.03779584914445877\n",
      "195 0.03643067181110382\n",
      "196 0.035117682069540024\n",
      "197 0.03385406360030174\n",
      "198 0.03263778239488602\n",
      "199 0.03146982938051224\n",
      "200 0.030345356091856956\n",
      "201 0.029262304306030273\n",
      "202 0.028220562264323235\n",
      "203 0.02721833437681198\n",
      "204 0.02625354565680027\n",
      "205 0.025323664769530296\n",
      "206 0.02442932315170765\n",
      "207 0.023567935451865196\n",
      "208 0.022739168256521225\n",
      "209 0.021939238533377647\n",
      "210 0.0211693923920393\n",
      "211 0.020427796989679337\n",
      "212 0.01971343718469143\n",
      "213 0.0190261360257864\n",
      "214 0.01836295984685421\n",
      "215 0.017724119126796722\n",
      "216 0.017108308151364326\n",
      "217 0.016515403985977173\n",
      "218 0.015943557024002075\n",
      "219 0.015392407774925232\n",
      "220 0.01486111432313919\n",
      "221 0.014349011704325676\n",
      "222 0.013855252414941788\n",
      "223 0.013379251584410667\n",
      "224 0.012920645996928215\n",
      "225 0.012478447519242764\n",
      "226 0.012052124366164207\n",
      "227 0.01164051704108715\n",
      "228 0.011243665590882301\n",
      "229 0.010860526002943516\n",
      "230 0.010491489432752132\n",
      "231 0.01013527438044548\n",
      "232 0.009791805408895016\n",
      "233 0.009460470639169216\n",
      "234 0.009141030721366405\n",
      "235 0.00883263535797596\n",
      "236 0.008535141125321388\n",
      "237 0.008247914724051952\n",
      "238 0.007971054874360561\n",
      "239 0.007703827694058418\n",
      "240 0.007445728871971369\n",
      "241 0.0071970876306295395\n",
      "242 0.00695680221542716\n",
      "243 0.006724654231220484\n",
      "244 0.006500610616058111\n",
      "245 0.006284444127231836\n",
      "246 0.006075821816921234\n",
      "247 0.005874521564692259\n",
      "248 0.0056800213642418385\n",
      "249 0.00549229746684432\n",
      "250 0.005310957785695791\n",
      "251 0.005136037245392799\n",
      "252 0.004966908134520054\n",
      "253 0.004803669173270464\n",
      "254 0.004645862616598606\n",
      "255 0.004493494983762503\n",
      "256 0.004346322268247604\n",
      "257 0.004204175900667906\n",
      "258 0.004066833760589361\n",
      "259 0.003934163134545088\n",
      "260 0.00380604132078588\n",
      "261 0.0036821861285716295\n",
      "262 0.0035626154858618975\n",
      "263 0.0034472234547138214\n",
      "264 0.00333566777408123\n",
      "265 0.0032277971040457487\n",
      "266 0.003123413072898984\n",
      "267 0.0030225825030356646\n",
      "268 0.002925192005932331\n",
      "269 0.002831005724146962\n",
      "270 0.0027399638202041388\n",
      "271 0.0026520411483943462\n",
      "272 0.0025669739115983248\n",
      "273 0.0024848191533237696\n",
      "274 0.0024053200613707304\n",
      "275 0.0023286384530365467\n",
      "276 0.0022544681560248137\n",
      "277 0.0021827046293765306\n",
      "278 0.0021132994443178177\n",
      "279 0.0020462116226553917\n",
      "280 0.0019813268445432186\n",
      "281 0.0019186127465218306\n",
      "282 0.0018579891184344888\n",
      "283 0.0017993446672335267\n",
      "284 0.0017425855621695518\n",
      "285 0.0016876806039363146\n",
      "286 0.0016345654148608446\n",
      "287 0.0015832741046324372\n",
      "288 0.0015336177311837673\n",
      "289 0.0014856174821034074\n",
      "290 0.0014391320291906595\n",
      "291 0.0013941406505182385\n",
      "292 0.0013506138930097222\n",
      "293 0.0013085290556773543\n",
      "294 0.0012677880004048347\n",
      "295 0.0012283530086278915\n",
      "296 0.0011901911348104477\n",
      "297 0.0011532703647390008\n",
      "298 0.0011175390100106597\n",
      "299 0.0010829915991052985\n",
      "300 0.001049602054990828\n",
      "301 0.001017285860143602\n",
      "302 0.00098600541241467\n",
      "303 0.0009557157172821462\n",
      "304 0.0009263587417080998\n",
      "305 0.000897978781722486\n",
      "306 0.0008704939391463995\n",
      "307 0.0008439102675765753\n",
      "308 0.0008181545417755842\n",
      "309 0.0007932201842777431\n",
      "310 0.0007690549246035516\n",
      "311 0.0007456818129867315\n",
      "312 0.0007230113260447979\n",
      "313 0.0007010839763097465\n",
      "314 0.0006798563990741968\n",
      "315 0.000659305602312088\n",
      "316 0.0006393603980541229\n",
      "317 0.0006200542557053268\n",
      "318 0.0006013419479131699\n",
      "319 0.0005832261522300541\n",
      "320 0.000565681082662195\n",
      "321 0.0005486804293468595\n",
      "322 0.0005322110373526812\n",
      "323 0.000516265572514385\n",
      "324 0.0005008223815821111\n",
      "325 0.00048584837350063026\n",
      "326 0.0004713374364655465\n",
      "327 0.00045727784163318574\n",
      "328 0.00044363588676787913\n",
      "329 0.0004304357571527362\n",
      "330 0.0004176436923444271\n",
      "331 0.00040522421477362514\n",
      "332 0.00039320296491496265\n",
      "333 0.000381550460588187\n",
      "334 0.0003702575631905347\n",
      "335 0.00035931591992266476\n",
      "336 0.00034871214302256703\n",
      "337 0.00033842228003777564\n",
      "338 0.00032843617373146117\n",
      "339 0.0003187891561537981\n",
      "340 0.00030941059230826795\n",
      "341 0.00030032030190341175\n",
      "342 0.0002915087970905006\n",
      "343 0.0002829617587849498\n",
      "344 0.00027468166081234813\n",
      "345 0.0002666517102625221\n",
      "346 0.0002588688221294433\n",
      "347 0.00025132179143838584\n",
      "348 0.00024398998357355595\n",
      "349 0.00023688978399150074\n",
      "350 0.0002299969783052802\n",
      "351 0.00022332485241349787\n",
      "352 0.00021684767853002995\n",
      "353 0.0002105585008393973\n",
      "354 0.0002044678694801405\n",
      "355 0.00019854902348015457\n",
      "356 0.0001928111450979486\n",
      "357 0.00018724559049587697\n",
      "358 0.00018185196677222848\n",
      "359 0.00017661017773207277\n",
      "360 0.00017153177759610116\n",
      "361 0.00016660746769048274\n",
      "362 0.00016182051331270486\n",
      "363 0.00015718959912192076\n",
      "364 0.00015267936396412551\n",
      "365 0.0001483117084717378\n",
      "366 0.00014407013077288866\n",
      "367 0.00013995783228892833\n",
      "368 0.00013596212374977767\n",
      "369 0.0001320836745435372\n",
      "370 0.00012832431821152568\n",
      "371 0.00012467731721699238\n",
      "372 0.00012113017146475613\n",
      "373 0.00011769653065130115\n",
      "374 0.00011435784108471125\n",
      "375 0.00011111830826848745\n",
      "376 0.00010797605500556529\n",
      "377 0.00010492341971257702\n",
      "378 0.00010196021321462467\n",
      "379 9.908171341521665e-05\n",
      "380 9.62893755058758e-05\n",
      "381 9.357751696370542e-05\n",
      "382 9.09473528736271e-05\n",
      "383 8.839202928356826e-05\n",
      "384 8.590965444454923e-05\n",
      "385 8.350054849870503e-05\n",
      "386 8.116184471873567e-05\n",
      "387 7.889181142672896e-05\n",
      "388 7.668322359677404e-05\n",
      "389 7.454047590726987e-05\n",
      "390 7.246099994517863e-05\n",
      "391 7.044145604595542e-05\n",
      "392 6.847554323030636e-05\n",
      "393 6.656948244199157e-05\n",
      "394 6.472023960668594e-05\n",
      "395 6.292137550190091e-05\n",
      "396 6.117722659837455e-05\n",
      "397 5.948257603449747e-05\n",
      "398 5.783290180261247e-05\n",
      "399 5.6237120588775724e-05\n",
      "400 5.46783376194071e-05\n",
      "401 5.316698661772534e-05\n",
      "402 5.170224540052004e-05\n",
      "403 5.027523366152309e-05\n",
      "404 4.889153206022456e-05\n",
      "405 4.7547022404614836e-05\n",
      "406 4.6238892537076026e-05\n",
      "407 4.496948895393871e-05\n",
      "408 4.373498813947663e-05\n",
      "409 4.253659790265374e-05\n",
      "410 4.137160431127995e-05\n",
      "411 4.024025110993534e-05\n",
      "412 3.913695763912983e-05\n",
      "413 3.806974564213306e-05\n",
      "414 3.702849426190369e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "415 3.602004289859906e-05\n",
      "416 3.5039814974879846e-05\n",
      "417 3.4086002415278926e-05\n",
      "418 3.3158474252559245e-05\n",
      "419 3.2257161365123466e-05\n",
      "420 3.1381292501464486e-05\n",
      "421 3.053232649108395e-05\n",
      "422 2.9703087420784868e-05\n",
      "423 2.8898906748509035e-05\n",
      "424 2.8116905014030635e-05\n",
      "425 2.735624548222404e-05\n",
      "426 2.6618710762704723e-05\n",
      "427 2.589946052466985e-05\n",
      "428 2.5200133677572012e-05\n",
      "429 2.4521566956536844e-05\n",
      "430 2.3861784939072095e-05\n",
      "431 2.3221768060466275e-05\n",
      "432 2.259662505821325e-05\n",
      "433 2.1989380911691114e-05\n",
      "434 2.139976277248934e-05\n",
      "435 2.0825544197577983e-05\n",
      "436 2.026962101808749e-05\n",
      "437 1.972840618691407e-05\n",
      "438 1.9201726900064386e-05\n",
      "439 1.868485014711041e-05\n",
      "440 1.8187707610195503e-05\n",
      "441 1.77027250174433e-05\n",
      "442 1.7231966921826825e-05\n",
      "443 1.677339787420351e-05\n",
      "444 1.6325964679708704e-05\n",
      "445 1.589220300957095e-05\n",
      "446 1.5469682693947107e-05\n",
      "447 1.505804084445117e-05\n",
      "448 1.4659090084023774e-05\n",
      "449 1.4270526662585326e-05\n",
      "450 1.3891894923290238e-05\n",
      "451 1.3524968380806968e-05\n",
      "452 1.316844645771198e-05\n",
      "453 1.2820334632124286e-05\n",
      "454 1.2481828889576718e-05\n",
      "455 1.2151404916949105e-05\n",
      "456 1.1831169103970751e-05\n",
      "457 1.151983997260686e-05\n",
      "458 1.1216853636142332e-05\n",
      "459 1.0921164175670128e-05\n",
      "460 1.0634528734954074e-05\n",
      "461 1.0355754966440145e-05\n",
      "462 1.0084525456477422e-05\n",
      "463 9.818895705393516e-06\n",
      "464 9.561628758092411e-06\n",
      "465 9.311656867794227e-06\n",
      "466 9.06800869415747e-06\n",
      "467 8.83038137544645e-06\n",
      "468 8.600522050983272e-06\n",
      "469 8.37572861200897e-06\n",
      "470 8.157939191733021e-06\n",
      "471 7.94417519500712e-06\n",
      "472 7.737810847174842e-06\n",
      "473 7.536523753515212e-06\n",
      "474 7.341115178860491e-06\n",
      "475 7.148817985580536e-06\n",
      "476 6.964214662730228e-06\n",
      "477 6.783450317016104e-06\n",
      "478 6.607127488678088e-06\n",
      "479 6.4358955569332466e-06\n",
      "480 6.269143341341987e-06\n",
      "481 6.106336059019668e-06\n",
      "482 5.949293154117186e-06\n",
      "483 5.795768174721161e-06\n",
      "484 5.645460987580009e-06\n",
      "485 5.500284260051558e-06\n",
      "486 5.358379894460086e-06\n",
      "487 5.219836111791665e-06\n",
      "488 5.085677912575193e-06\n",
      "489 4.954251835442847e-06\n",
      "490 4.827542852581246e-06\n",
      "491 4.702882961282739e-06\n",
      "492 4.5827068788639735e-06\n",
      "493 4.464645826374181e-06\n",
      "494 4.350607468950329e-06\n",
      "495 4.238191650074441e-06\n",
      "496 4.129782155359862e-06\n",
      "497 4.024288955406519e-06\n",
      "498 3.921175903087715e-06\n",
      "499 3.820653546426911e-06\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold inputs and outputs, and wrap them in Variables.\n",
    "x = Variable(torch.randn(N, D_in))\n",
    "y = Variable(torch.randn(N, D_out), requires_grad=False)\n",
    "\n",
    "# Use the nn package to define our model as a sequence of layers.\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    ")\n",
    "\n",
    "# The nn package also contains definitions of popular loss functions\n",
    "loss_fn = torch.nn.MSELoss(size_average=False)\n",
    "\n",
    "learning_rate = 1e-4\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y by passing x to the model.\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # Compute and print loss. \n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Zero the gradients before running the backward pass.\n",
    "    model.zero_grad()\n",
    "\n",
    "    # Backward pass\n",
    "    loss.backward()\n",
    "\n",
    "    # Update the weights using gradient descent.\n",
    "    for param in model.parameters():\n",
    "        param.data -= learning_rate * param.grad.data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
